#coding:utf-8
import scrapy
import json
import csvout

class QuotesSpider(scrapy.Spider):
    name = "iteye"

    def start_requests(self):
        url = 'http://www.iteye.com'
        tag = getattr(self, 'tag', None)

        if tag is not None:
            url = url + '/search?query=' + tag + '&type=blog'
            yield scrapy.Request(url, callback=self.parse)

    def parse(self, response):
        #从chrome里拷出来的xpath//*[@id="search_result"]/div[13]/a[3]
        res = response.xpath('//*[@id="search_result"]/div[13]/a[last()-1]/text()').extract_first()
        #如果结果只有一页
        if res is None:
            response.meta['alnum'] = 0
            yield self.parse_last_page(response)
            #yield scrapy.Request(response.url, callback=self.parse_last_page, meta={'alnum': 0})
        #如果有多页，先统计页数（每页10个），再统计最后一页
        else:
            num = filter(lambda x: x.isdigit(), res)
            num = int(num)
            num = num * 10
            last_page = response.xpath('//*[@id="search_result"]/div[13]/a[last()-1]/@href').extract_first()
            yield scrapy.Request(response.urljoin(last_page), callback=self.parse_last_page, meta={'alnum': num})
        
    def parse_last_page(self, response):
        res = response.xpath('//*[@id="search_result"]/div/div/h4').extract()
        num = len(res)
        num = num + response.meta['alnum']
        outdic = {
            'iteye_num': num
        }
        outli = ['iteye_num', num, response.url]
        csvout.list2csv(outli)
        #此处使用yield会导致结果只有一页时报错，因为返回的是generator而不是dict
        return outdic
